From: Jeroen van der Heijden Date: Wed, 3 Oct 2018 21:06:17 +0000 (+0200) Subject: Re-write buffer X-Git-Tag: archive/raspbian/2.0.44-1+rpi1~1^2~3^2~8^2~41 X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/%22/%22http:/www.example.com/cgi/%22?a=commitdiff_plain;h=54b12fd0b94eef89f64b2ad7bb37c07c945a4a9a;p=siridb-server.git Re-write buffer --- diff --git a/include/siri/db/buffer.h b/include/siri/db/buffer.h index 8b4ca624..326c393a 100644 --- a/include/siri/db/buffer.h +++ b/include/siri/db/buffer.h @@ -19,16 +19,18 @@ typedef struct siridb_buffer_s siridb_buffer_t; #include #include -#define MAX_BUFFER_SZ 10485760 - +#define MAX_BUFFER_SZ 1048576 siridb_buffer_t * siridb_buffer_new(void); void siridb_buffer_free(siridb_buffer_t * buffer); +_Bool siridb_buffer_is_valid_size(ssize_t ssize); +void siridb_buffer_set_path(siridb_buffer_t * buffer, const char * str); int siridb_buffer_new_series( siridb_buffer_t * buffer, siridb_series_t * series); int siridb_buffer_open(siridb_buffer_t * buffer); int siridb_buffer_load(siridb_t * siridb); +int siridb_buffer_test_path(siridb_t * siridb); int siridb_buffer_write_empty( siridb_buffer_t * buffer, siridb_series_t * series); diff --git a/include/siri/db/points.h b/include/siri/db/points.h index 96863065..37dd6a19 100644 --- a/include/siri/db/points.h +++ b/include/siri/db/points.h @@ -32,6 +32,7 @@ typedef struct siridb_points_s siridb_points_t; void siridb_points_init(void); siridb_points_t * siridb_points_new(size_t size, points_tp tp); void siridb_points_free(siridb_points_t * points); +int siridb_points_resize(siridb_points_t * points, size_t n); void siridb_points_add_point( siridb_points_t *__restrict points, uint64_t * ts, diff --git a/src/siri/admin/request.c b/src/siri/admin/request.c index 9b278472..02f1710e 100644 --- a/src/siri/admin/request.c +++ b/src/siri/admin/request.c @@ -587,7 +587,7 @@ static cproto_server_t ADMIN_on_new_database( buffer_size = (qp_buffer_size.tp == QP_HOOK) ? DEFAULT_BUFFER_SIZE : qp_buffer_size.via.int64; - if (buffer_size % 512 || buffer_size < 512 || buffer_size > MAX_BUFFER_SZ) + if (!siridb_buffer_is_valid_size(buffer_size)) { sprintf(err_msg, "invalid buffer size: %" PRId64 diff --git a/src/siri/db/buffer.c b/src/siri/db/buffer.c index 3e5d6b1f..d1281a6c 100644 --- a/src/siri/db/buffer.c +++ b/src/siri/db/buffer.c @@ -9,6 +9,9 @@ * - initial version, 01-04-2016 * */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif #include #include #include @@ -55,7 +58,7 @@ siridb_buffer_t * siridb_buffer_new(void) buffer->fd = 0; buffer->fp = NULL; buffer->len = 0; - buffer->nsize = 0; + buffer->nsize = 0; /* 0 means no new size */ buffer->path = NULL; buffer->size = 0; buffer->template = NULL; @@ -75,6 +78,50 @@ void siridb_buffer_free(siridb_buffer_t * buffer) free(buffer); } +_Bool siridb_buffer_is_valid_size(ssize_t ssize) +{ + return ssize >= 512 && (ssize % 512) == 0 && ssize <= MAX_BUFFER_SZ; +} + +void siridb_buffer_set_path(siridb_buffer_t * buffer, const char * str) +{ + size_t lstr = strlen(str); + size_t lf = strlen(SIRIDB_BUFFER_FN); // size of "buffer.dat" + size_t lspf = strlen("/" SIRIDB_BUFFER_FN); // size of "/buffer.dat" + assert (buffer->path == NULL); + + if (str[lstr-1] == '/') + { + buffer->path = strdup(str); + } + else if (lstr >= lspf && strcmp(str+lstr-lspf, "/" SIRIDB_BUFFER_FN) == 0) + { + buffer->path = strndup(str, lstr-lf); + } + else if (asprintf(&buffer->path, "%s/", str) < 0) + { + free(buffer->path); + buffer->path = NULL; + } + + if (buffer->path == NULL) + { + log_critical("Allocation error while setting buffer path"); + return; + } +} + +int siridb_buffer_test_path(siridb_t * siridb) +{ + siridb_misc_get_fn(fn, siridb->buffer->path, SIRIDB_BUFFER_FN) + if (siridb->series_map->len && !xpath_file_exist(fn)) + { + log_critical("Cannot read buffer file: '%s'", fn); + return -1; + } + return 0; +} + /* * Returns 0 if success or EOF in case of an error. */ @@ -215,9 +262,9 @@ int siridb_buffer_load(siridb_t * siridb) siridb_buffer_t * buffer = siridb->buffer; FILE * fp; FILE * fp_temp; - size_t read_at_once = 8; + size_t read_at_once = (size_t) (MAX_BUFFER_SZ / buffer->size); size_t num, i; - char buf[buffer->size * read_at_once]; + char * buf; char * pt, * end; long int offset = 0; siridb_series_t * series; @@ -227,9 +274,11 @@ int siridb_buffer_load(siridb_t * siridb) log_info("Loading and cleanup buffer"); + buf = malloc(read_at_once * buffer->size); buffer->template = malloc(buffer->size); - if (buffer->template == NULL) + if (buf == NULL || buffer->template == NULL) { + free(buf); log_critical("Allocation error while loading buffer"); return -1; } @@ -249,6 +298,7 @@ int siridb_buffer_load(siridb_t * siridb) if (xpath_file_exist(fn_temp)) { + free(buf); log_error( "Temporary buffer file found: '%s'. " "Check if something went wrong or remove this file", fn_temp); @@ -257,13 +307,8 @@ int siridb_buffer_load(siridb_t * siridb) if ((fp = fopen(fn, "r")) == NULL) { - if (siridb->series_map->len) - { - log_critical("Buffer file '%s' not found.", fn); - return -1; - } - log_warning("Buffer file '%s' not found, create a new one.", fn); - + free(buf); + log_info("Buffer file '%s' not found, create a new one.", fn); if ((fp = fopen(fn, "w")) == NULL) { log_critical("Cannot create buffer file '%s'.", fn); @@ -276,6 +321,7 @@ int siridb_buffer_load(siridb_t * siridb) { log_critical("Cannot open '%s' for writing", fn_temp); fclose(fp); + free(buf); return -1; } @@ -314,7 +360,7 @@ int siridb_buffer_load(siridb_t * siridb) series->id); fclose(fp); fclose(fp_temp); - + free(buf); return -1; /* signal is raised */ } @@ -338,12 +384,14 @@ int siridb_buffer_load(siridb_t * siridb) fn_temp); fclose(fp); fclose(fp_temp); - + free(buf); return -1; } } } + free(buf); + if (fclose(fp) || fclose(fp_temp) || rename(fn_temp, fn)) diff --git a/src/siri/db/db.c b/src/siri/db/db.c index 3e94fe8d..fc2d3b97 100644 --- a/src/siri/db/db.c +++ b/src/siri/db/db.c @@ -184,6 +184,14 @@ siridb_t * siridb_new(const char * dbpath, int lock_flags) return NULL; } + /* test buffer path */ + if (siridb_buffer_test_path(siridb)) + { + log_error("Cannot read buffer for database '%s'", siridb->dbname); + siridb_decref(siridb); + return NULL; + } + /* load shards */ if (siridb_shards_load(siridb)) { @@ -372,8 +380,7 @@ static int siridb__from_unpacker( /* read buffer size, same buffer_size requirements are used in request.c */ if ( qp_next(unpacker, &qp_obj) != QP_INT64 || - qp_obj.via.int64 % 512 || - qp_obj.via.int64 < 512) + !siridb_buffer_is_valid_size(qp_obj.via.int64)) { READ_DB_EXIT_WITH_ERROR("Cannot read buffer size.") } @@ -835,35 +842,30 @@ static int siridb__read_conf(siridb_t * siridb) /* read buffer_path from database.conf */ rc = cfgparser_get_option(&option, cfgparser, "buffer", "path"); - if (rc == CFGPARSER_SUCCESS && option->tp == CFGPARSER_TP_STRING) + siridb_buffer_set_path( + buffer, + (rc == CFGPARSER_SUCCESS && option->tp == CFGPARSER_TP_STRING) ? + option->val->string : siridb->dbpath); + + /* read buffer size from database.conf */ + rc = cfgparser_get_option(&option, cfgparser, "buffer", "size"); + if (rc == CFGPARSER_SUCCESS && option->tp == CFGPARSER_TP_INTEGER) { - size_t len = strlen(option->val->string); - buffer->path = NULL; - if (option->val->string[len - 1] == '/') - { - buffer->path = strdup(option->val->string); - } - else if ( - len >= 11 && - strcmp(option->val->string + (len-11), "/buffer.dat") == 0) + ssize_t ssize = option->val->integer; + if (!siridb_buffer_is_valid_size(ssize)) { - buffer->path = strndup(option->val->string, len-10); + log_warning( + "Invalid buffer size: %" PRId64 + " (expecting a multiple of 512 with a maximum of %" PRId64 ")", + ssize, + (int64_t) MAX_BUFFER_SZ); } - else if (asprintf(&buffer->path, "%s/", option->val->string) < 0) + else { - buffer->path = NULL; + buffer->nsize = (buffer->size == (size_t) ssize) ? + 0 : (size_t) ssize; } } - else - { - buffer->path = strdup(siridb->dbpath); - } - - rc = cfgparser_get_option(&option, cfgparser, "buffer", "size"); - if (rc == CFGPARSER_SUCCESS && option->tp == CFGPARSER_TP_INTEGER) - { - - } cfgparser_free(cfgparser); diff --git a/src/siri/db/points.c b/src/siri/db/points.c index 8ce32992..c78b9c8b 100644 --- a/src/siri/db/points.c +++ b/src/siri/db/points.c @@ -98,6 +98,21 @@ siridb_points_t * siridb_points_new(size_t size, points_tp tp) return points; } +/* + * Resize points to a new size. Returns 0 when successful or -1 if failed. + */ +int siridb_points_resize(siridb_points_t * points, size_t n) +{ + assert( points->len <= n ); + siridb_point_t * tmp = realloc(points->data, sizeof(siridb_point_t) * n); + if (tmp == NULL && n) + { + return -1; + } + points->data = tmp; + return 0; +} + /* * Returns a copy of points or NULL in case of an error. NULL is also returned * if points is NULL. diff --git a/src/siri/db/series.c b/src/siri/db/series.c index 2e3d9db3..7c5adde1 100644 --- a/src/siri/db/series.c +++ b/src/siri/db/series.c @@ -761,27 +761,10 @@ siridb_points_t * siridb_series_get_points( } } - if (points->len < size) + if (points->len < size && siridb_points_resize(points, points->len)) { - /* shrink allocation size */ - point = (siridb_point_t *) - realloc(points->data, points->len * sizeof(siridb_point_t)); - if (point == NULL && points->len) - { - log_error("Re-allocation points has failed"); - } - else - { - points->data = point; - } + log_error("Re-allocation points has failed"); } -#if DEBUG - else - { - /* size must be equal if not smaller */ - assert (points->len == size); - } -#endif return points; }